# -*- coding: utf-8 -*-
import sys
sys.path.append("..")

from MysqlCon import MysqlCon
from pymongo import MongoClient
import pymysql
import time


class MongoReader(object):
  """
  """


  #多行插入
  def insert_many(self, connect, cursor, insert_sql, insert_data):
    connect.ping(reconnect = True)

    # 批量插入
    try:
      res = cursor.executemany(insert_sql, insert_data)
      print("res = ", res)
      # 提交，不进行提交无法保存到数据库
      connect.commit()
    except Exception as e:
      print(e)
      connect.rollback()
    #finally:
    #connect.close()

    return True

  #写入文件
  def insert_file(self, fileName, dataList):
    f = "d:\\data\\mongodb\\%s.txt" %(fileName)
    try:
      #只需要将之前的”w"改为“a"即可，代表追加内容
      with open(f, "a" ,encoding="utf-8") as file:
        for json in dataList:
          file.write(str(json) + "\n")
    except Exception as e:
      print(e)

    return True

  #写入读取错误的标签名
  def write_error_tags(self, data):
    f = "d:\\data\\mongodb\\error_tags.txt"
    try:
      with open(f, "a" ,encoding="utf-8") as file:
        file.write(data + "\n")
    except Exception as e:
      print(e)

    return True


  #读单体标签
  def read_person(self, mycol, ks, imeiList, otherList):
    # 循环读取数据
    for x in mycol.find():
      '''keys = x.keys()
      kl = len(keys)
      keyStr = str(keys)
      if not (keyStr in ks) :
          ks.add(keyStr)
          print(keyStr)
          print(x)
          print('person_tags===========', kl)
      continue'''
      try:
        #有sImei值的入库：dict_keys(['_id', 'data', 'order_id', 'mission_id', 'sImei', 'iStatus', 'app_id'])
        sImei=x['sImei']
        _id=str(x['_id'])
        dataList=x['data']
        print(len(dataList))
        for data in dataList:
          tag_id=data['tagid']
          tag_value=data['tag_value']
          print(tag_id, tag_value)
        break
        order_id=x['order_id']
        mission_id=x['mission_id']
        iStatus=x['iStatus']
        try:
          app_id=x['app_id']
        except Exception as e:
          #print("exception：",e)
          app_id=''
        par = (_id, data, order_id, mission_id, sImei, iStatus, app_id)
        imeiList.append(par)
      except Exception as e:
        #没有sImei值的入文件
        otherList.append(x)


    return ks


  #读群体标签
  def read_group(self, mycol, ks, gList):
    # 循环读取数据
    for x in mycol.find():
      #print(x)
      '''keys = x.keys()
      kl = len(keys)
      keyStr = str(keys)
      if not (keyStr in ks) :
          ks.add(keyStr)
          print(keyStr)
          print(x)
          print('group_tags########', kl)
      continue'''
      #dict_keys(['_id', 'data', 'order_id', 'mission_id', 'sTagID', 'iStatus', 'app_id'])
      _id=str(x['_id'])
      data=str(x['data'])
      order_id=x['order_id']
      mission_id=x['mission_id']
      sTagID=x['sTagID']
      iStatus=x['iStatus']
      try:
        app_id=x['app_id']
      except Exception as e:
        #print("exception：",e)
        app_id=''
      par = (_id, data, order_id, mission_id, sTagID, iStatus, app_id)
      gList.append(par)
    return ks



if __name__ == "__main__":
  dict = {'Name': 'Runoob', 'Age': 7, 'Class': 'First'}

  #mongo
  client = MongoClient("mongodb://root:root@172.28.56.90:27017/admin")
  goingdata = client["goingdata"]

  #mysql
  mysqlConn = pymysql.connect(host='172.28.56.90', user='mysql', password='egSQ7HhxajHZjvdX', port=3306, database='odsdb',charset="utf8")
  cursor = mysqlConn.cursor()

  #记录执行开始时间
  str_time = int(time.time())

  #实例化类
  reader = MongoReader()

  # 读取原文件
  collist = goingdata. list_collection_names()
  maxRow = len(collist)
  print("总记录数", maxRow)

  row = 0
  pks, gks = {'0'}, {'0'}
  imeiList, otherList, gList = [], [], []
  jg_person_tags_07_sql = """
        insert into jg_person_tags_07(
            _id, data, order_id, mission_id, sImei, iStatus, app_id
            ) VALUES (%s, %s, %s, %s, %s, %s, %s)
        """
  jg_group_tags_sql = """
        insert into jg_group_tags(
            _id, data, order_id, mission_id, sTagID, iStatus, app_id
            ) VALUES (%s, %s, %s, %s, %s, %s, %s)
        """
  for col in collist:
    #print(col)
    #break
    mycol = goingdata[col]
    if col.find('group_tags') == 0 : #群体标签
      continue
      try:
        ks = reader.read_group(mycol, gks, gList)
      except Exception as e:
        reader.write_error_tags("%s : %s" %(col, e))

    elif col.find('person_tags') == 0 :#单体标签
      #continue
      try:
        ks = reader.read_person(mycol, pks, imeiList, otherList)
      except Exception as e:
        reader.write_error_tags("%s : %s" %(col, e))
    else :
      continue
    break
    row += 1
    if row > 0 and (row % 100 == 0 or row == maxRow):
      # 批量写入数据
      if len(imeiList) > 0 :
        print("imeiList@@@@@@#", len(imeiList))
        reader.insert_many(mysqlConn, cursor, jg_person_tags_07_sql, imeiList)
        imeiList.clear()
      if len(otherList) > 1000 :
        print("otherList@@@@@@#", len(otherList))
        reader.insert_file(col, otherList)
        #print(otherList)
        otherList.clear()
      if len(gList) > 0 :
        print("gList@@@@@@#", len(gList))
        reader.insert_many(mysqlConn, cursor, jg_group_tags_sql, gList)
        gList.clear()

    #if row == 10000 :
    #break

  mysqlConn.close()
  client.close()

  end_time = int(time.time())
  print("程序执行时间（秒）：", end_time - str_time)
